﻿//  Symplectic Spark
//  Copyright 2010 Symplectic Ltd
//  Created by Martyn Whitwell (martyn@symplectic.co.uk)

//  This file is part of Spark.

//  Spark is free software: you can redistribute it and/or modify
//  it under the terms of the GNU General Public License as published by
//  the Free Software Foundation, either version 3 of the License, or
//  (at your option) any later version.

//  Spark is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//  GNU General Public License for more details.

//  You should have received a copy of the GNU General Public License
//  along with Spark.  If not, see <http://www.gnu.org/licenses/>.

// $URL: https://symplectic-spark.googlecode.com/svn/trunk/Symplectic.Spark.UserFeedIndex/HackedUserFeed.cs $
// $LastChangedDate: 2010-03-31 12:37:14 +0100 (Wed, 31 Mar 2010) $
// $LastChangedRevision: 31 $
// $LastChangedBy: martyn@symplectic.co.uk $

using System;
using System.Configuration;
using System.Linq;
using System.Web;
using System.Diagnostics;
using System.IO;
using System.Net;
using System.Xml;
using System.Xml.Linq;
using System.Collections.Generic;

using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.Index;
using Lucene.Net.Search;
using Lucene.Net.Documents;
using Lucene.Net.QueryParsers;


/// <summary>
/// Summary description for HackUserFeed
/// </summary>

namespace Symplectic.Spark
{
    public class HackedUserFeed
    {
        #region Logger
        private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
        #endregion
        
        #region Constants
        private const int PERPAGE = 100;
        private static XNamespace API_NAMESPACE = "http://www.symplectic.co.uk/publications/api";
        private static XNamespace ATOM_NAMESPACE = "http://www.w3.org/2005/Atom";
        #endregion

        #region Private members
        private string apiBaseUri;
        private IndexModifier IndexModifier { get; set; }
        private string IndexDirectory { get; set; }
        #endregion
        
        #region Constructors
        public HackedUserFeed(string indexDirectory)
        {
            this.IndexDirectory = indexDirectory;
        }

        public HackedUserFeed(string indexDirectory, string apiBaseUri)
        {
            this.apiBaseUri = apiBaseUri;
            this.IndexDirectory = indexDirectory;

            // initialise the index, creating it if it does not already exist
            IndexModifier = new IndexModifier(
                IndexDirectory,
                new Lucene.Net.Analysis.Standard.StandardAnalyzer(),
                !Directory.Exists(IndexDirectory)
                );
        }
        #endregion
        
        #region API Methods

        private XElement ReadAPIData(string url)
        {

            WebRequest apiRequest = HttpWebRequest.Create(url);
            WebResponse apiResponse = null;
            try
            {
                apiResponse = apiRequest.GetResponse();
            }
            catch (WebException exc)
            {
                if (exc.Status == WebExceptionStatus.NameResolutionFailure)
                    throw new Exception("Could not resolve: " + url, exc);
                else
                    throw new WebException("Failed to retrieve API data from " + apiRequest.RequestUri.AbsoluteUri, exc);
            }

            Stream apiResponseStream = apiResponse.GetResponseStream();
            XmlReader apiReader = XmlReader.Create(apiResponseStream);
            while (apiReader.Read() && apiReader.NodeType != XmlNodeType.Element) ; //read until you get to an element
            return (XElement)XNode.ReadFrom(apiReader);
        }


        public int UpdateUsersFromAPI()
        {
            int thisPage = 0;
            int nextPage = 1; // start here
            int lastPage = 0;
            string url;
            int count = 0;

            //loop through the pages of related object
            do
            {
               // XElement users = new XElement("users");

                url = String.Format("{0}objects?categories={1}&detail={2}&per-page={3}&page={4}",
                    apiBaseUri, "users", "full", PERPAGE, nextPage);

                //Load XML data
                XElement apiData = ReadAPIData(url);

                //Reset pagination information
                thisPage = 0;
                nextPage = 0;
                lastPage = 0;
                //Get pagination information
                XElement pagination = apiData.Element(API_NAMESPACE + "pagination");
                foreach (XElement page in pagination.Elements(API_NAMESPACE + "page"))
                {
                    switch (page.Attribute("position").Value)
                    {
                        case "this":
                            thisPage = int.Parse(page.Attribute("number").Value);
                            break;
                        case "next":
                            nextPage = int.Parse(page.Attribute("number").Value);
                            break;
                        case "last":
                            lastPage = int.Parse(page.Attribute("number").Value);
                            break;
                    }
                }

                IEnumerable<XElement> apiObjects = apiData.Descendants(API_NAMESPACE + "object");
                foreach (XElement apiObject in apiObjects)
                {
                    UpdateIndexedElement(apiObject);
                    count++;
                }

                IndexModifier.Optimize();
                IndexModifier.Flush();

               // break;//TEMPORARY
            } while (nextPage > 0 && thisPage < lastPage && nextPage > thisPage);

            return count;
        }
        #endregion

        #region Lucene Indexing Methods
              
        private void InsertIndexedElement(XElement element)
        {
            Logger.DebugFormat("Inserting id {0} into index", element.Attribute("id").Value);
            IndexModifier.AddDocument(BuildDocument(element));
        }

        private void UpdateIndexedElement(XElement element)
        {
            Console.WriteLine("Updating {0}", element.Attribute("id").Value);
            DeleteIndexedElement(element);
            InsertIndexedElement(element);
        }

        private int DeleteIndexedElement(XElement element)
        {
            int test = IndexModifier.DeleteDocuments(new Term("id", element.Attribute("id").Value));
            Logger.DebugFormat("Deleted id {0} from index, returned {1}", element.Attribute("id").Value, test);
            return test;
        }


        private Document BuildDocument(XElement element)
        {
            //Convert xml <document> from database into Lucene document for indexing
            Document document = new Document();

            //Index attributes as untokenised, stored
            foreach (XAttribute attr in element.Attributes())
            {
                switch (attr.Name.LocalName)
                {
                    case "id":
                    case "proprietary-id":
                    case "username":
                    case "is-deleted":
                        Field field = new Field(attr.Name.LocalName, attr.Value.ToLower(), Field.Store.YES, Field.Index.UN_TOKENIZED);
                        field.SetBoost(1.5f);
                        document.Add(field);
                        break;

                    default:
                        break; //do nowt
                }
            }

            //Index elements as both untokenised and tokenised, boosting lastname over other names
            foreach (XElement childElement in element.Elements())
            {
                string childElementName = childElement.Name.LocalName.ToLower();
                switch (childElementName)
                {
                    case "title":
                    case "initials":
                    case "first-name":
                    case "known-as":
                    case "suffix":
                        Field field1 = new Field(childElementName, childElement.Value, Field.Store.YES, Field.Index.UN_TOKENIZED);
                        Field field2 = new Field(childElementName, childElement.Value, Field.Store.NO, Field.Index.TOKENIZED);
                        field1.SetBoost(1.1f); //boost untokenised version
                        document.Add(field1);
                        document.Add(field2);
                        break;
                    case "last-name":
                        Field field3 = new Field(childElementName, childElement.Value, Field.Store.YES, Field.Index.UN_TOKENIZED);
                        Field field4 = new Field(childElementName, childElement.Value, Field.Store.NO, Field.Index.TOKENIZED);
                        field3.SetBoost(1.3f); //boost untokenised  lastname
                        field4.SetBoost(1.2f); //boost tokenised  lastname
                        document.Add(field3);
                        document.Add(field4);
                        break;
                    case "organisation-defined-data":
                        Field field5 = new Field(childElementName, childElement.Value, Field.Store.NO, Field.Index.TOKENIZED);
                        document.Add(field5);
                        break;
                    default:
                        break; //do nowt
                }
            }
            return document;
        }
        #endregion

        #region Lucene Querying Methods
        public XDocument Find(string query, int pageSize, int pageNum)
        {
            if (pageSize < 1)
                throw new ArgumentException("Invalid pageSize, must be >= 1");
            if (pageNum < 1)
                throw new ArgumentException("Invalid pageNum, must be >= 1");
            if (query == null || query.Trim().Equals(String.Empty))
                throw new ArgumentNullException("The searchString should not be null");

            IndexSearcher idxs = new IndexSearcher(IndexDirectory);
            XDocument results = new XDocument();
            XElement searchResults = new XElement("users");
            results.Add(searchResults);

            BooleanQuery querySearch = new BooleanQuery();
            Analyzer analyzer = new StandardAnalyzer();

            
            querySearch.Add(new TermQuery(new Term("id", query)), BooleanClause.Occur.SHOULD);
            querySearch.Add(new TermQuery(new Term("proprietary-id", query)), BooleanClause.Occur.SHOULD);
            querySearch.Add(new TermQuery(new Term("username", query)), BooleanClause.Occur.SHOULD);

            querySearch.Add(new QueryParser("title", analyzer).Parse(query), BooleanClause.Occur.SHOULD);
            querySearch.Add(new QueryParser("initials", analyzer).Parse(query), BooleanClause.Occur.SHOULD);
            querySearch.Add(new QueryParser("first-name", analyzer).Parse(query), BooleanClause.Occur.SHOULD);
            querySearch.Add(new QueryParser("last-name", analyzer).Parse(query), BooleanClause.Occur.SHOULD);
            querySearch.Add(new QueryParser("known-as", analyzer).Parse(query), BooleanClause.Occur.SHOULD);
            querySearch.Add(new QueryParser("suffix", analyzer).Parse(query), BooleanClause.Occur.SHOULD);
            querySearch.Add(new QueryParser("organisation-defined-data", analyzer).Parse(query), BooleanClause.Occur.SHOULD);


            //filter out deleted users
            //querySearch.Add(new TermQuery(new Term("is-deleted", "false")), BooleanClause.Occur.MUST); //Only find users who are not deleted
            //queryGroups.Add(new TermQuery(new Term("group", group.ToString())), BooleanClause.Occur.SHOULD);

            //Get search hits
            Hits hits = idxs.Search(querySearch, new QueryFilter(new TermQuery(new Term("is-deleted", "false")))); //Only find users who are not deleted


            //Calculate pagination assuming specified page size
            int hitCount = hits.Length();
            int maxPage = (hitCount + pageSize - 1) / pageSize;
            if (maxPage < 1) maxPage = 1;

            int currentPage = (pageNum * pageSize > hitCount) ? maxPage : pageNum;
            searchResults.SetAttributeValue("count", hitCount);

            XElement pages = new XElement("pages",
                new XAttribute("current-page", currentPage),
                new XAttribute("max-page", maxPage),
                new XAttribute("page-size", pageSize));

            for (int p = 1; p <= maxPage; p++)
            {
                XElement page = new XElement("page", new XAttribute("number", p));

                if (p == currentPage)
                    page.SetAttributeValue("current", p == currentPage);

                int minResult = ((p - 1) * pageSize) + 1;
                int maxResult = (p * pageSize);
                if (hitCount == 0)
                    minResult = 0;
                if (maxResult > hitCount)
                    maxResult = hitCount;

                page.SetAttributeValue("title", String.Format("{0} - {1}", minResult, maxResult));

                pages.Add(page);
            }
            searchResults.Add(pages);



            //Print out results for current page
            for (int n = ((currentPage - 1) * pageSize); n < (currentPage * pageSize) && n < hitCount; n++)
            {
                Document hitDocument = hits.Doc(n);
                XElement user = new XElement("user", new XAttribute("score", hits.Score(n)));
                user.Add(new XAttribute("id", hits.Id(n)));
                user.Add(new XAttribute("number", n));

                foreach (Field f in hitDocument.Fields())
                {
                    switch (f.Name())
                    {
                        //case "data":
                       // case "html":
                            //do nothing - these values shouldn't be printed out in the result set
                       //     break;
                        default:
                            user.Add(new XElement(f.Name(), f.StringValue()));
                            break;
                    }
                }

                searchResults.Add(user);

                //i++;
            }
            return results;
        }
        #endregion
    }
}